package com.duowan.mspider.processor;

import com.duowan.mspider.download.MySeleniumDownloader;
import com.duowan.mspider.util.Constant.HtmlSite;
import com.duowan.mspider.util.FileUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.File;
import java.net.URLDecoder;
import java.util.List;

/**
 * html站点爬虫
 *
 * @author:wangyan1
 * @create_time: 2018-06-01 11:37
 */

public class HtmlSiteProcessor implements PageProcessor {

	private HtmlSite htmlSite;

	@Override
	public void process(Page page) {
		List<String> listUrls = page.getHtml().xpath(htmlSite.xpath).all();
		if (CollectionUtils.isEmpty(listUrls)) {
			return;
		}

		File file = new File(htmlSite.getSiteFileName());
		if (file.exists()) {
			file.delete();
		}

		for (String url : listUrls) {
			if (StringUtils.isBlank(url)) {
				continue;
			}
			try {
				String link = URLDecoder.decode(url, "UTF-8");
				if (StringUtils.isBlank(link)) {
					continue;
				}

				if (htmlSite.isRelativePath){
					link = htmlSite.url + link;
				}
				FileUtils.write(file, link);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
	}

	@Override
	public Site getSite() {
		return Site.me().setRetryTimes(3).setSleepTime(1000)
					   .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER");
	}

	public HtmlSiteProcessor(HtmlSite htmlSite) {
		this.htmlSite = htmlSite;
	}

	public HtmlSite getHtmlSite() {
		return htmlSite;
	}

	public static void main(String[] args) throws Exception {
		System.setProperty("selenuim_config", "config.ini");

		for (HtmlSite htmlSite : HtmlSite.values()) {
			HtmlSiteProcessor processor = new HtmlSiteProcessor(htmlSite);
			Spider spider = Spider.create(processor);
			spider.addUrl(htmlSite.url);
			//浏览器驱动（动态网页信息通过模拟浏览器启动获取）
			MySeleniumDownloader downloader = new MySeleniumDownloader("chromedriver.exe");
			spider.setDownloader(downloader).addPipeline(new ConsolePipeline()).thread(1).run();
		}
	}
}
